package fetcher

import (
	"net/http"
	"fmt"
	"golang.org/x/text/transform"
	"io/ioutil"
	"bufio"
	"golang.org/x/text/encoding"
	"golang.org/x/net/html/charset"
	"golang.org/x/text/encoding/unicode"
	"log"
	"time"
)

// 延迟1毫秒
var rateLimiter = time.Tick(1 * time.Millisecond)

/*转编码*/
func Fetch (url string )([]byte,error){

	// 每个goroutine 只有抢到这个rateLimiter 才能继续往下执行，否则一直等待着
	<- rateLimiter

	resp, err := http.Get(url)
	if err != nil {
		return nil,err
	}
	defer resp.Body.Close()
	if resp.StatusCode != http.StatusOK {
		return nil,fmt.Errorf("wrong status code: %d",resp.StatusCode)
	}
	/*转编码,转成utf8*/
	bodyReader := bufio.NewReader(resp.Body)
	encoding := determineEncoding(bodyReader)
	utf8Reader := transform.NewReader(bodyReader, encoding.NewDecoder())
	return ioutil.ReadAll(utf8Reader)
}
/*查询网页编码*/
func determineEncoding(r *bufio.Reader) encoding.Encoding{
	/*Peek 返回缓存的一个切片，在读取之前都是有效的*/
	bytes, err := r.Peek(1024)
	if err != nil {
		log.Printf("Fetcher error: %v",err)
		return unicode.UTF8
	}
	encoding, _, _ := charset.DetermineEncoding(bytes, "")
	return encoding
}

